# -*- coding: UTF-8 -*-
import scrapy
import messAround.util.help as util


# 百度热搜 top.baidu.com
# scrapy crawl baidu_top -o baidu.json
class BaiduTopSpider(scrapy.Spider):
    name = 'baidu_top'

    allowed_domains = ['top.baidu.com']

    start_urls = ['http://top.baidu.com/']

    custom_settings = {
        'ITEM_PIPELINES': {
            'messAround.pipeline.baidu.BaiduTopTopicPipeline': 300
        }
    }

    def parse_item(self, res):
        for index in range(1, 19):
            item_xpath = f'/html/body/div[1]/div/main/div[2]/div/div[2]/div[{str(index)}]'
            # 标题与连接
            title = res.xpath(f'{item_xpath}/div[2]/a/div[1]/text()').get()
            link = res.xpath(item_xpath + '/div[2]/a/@href').get()

            # 简介
            info = res.xpath(item_xpath + '/div[2]/div[2]/text()').get()

            # 搜索指数
            hot_index_xpath = item_xpath + '/div[1]/div[2]/text()'
            hot_index = res.xpath(hot_index_xpath).get()
            yield {
                'no': index,
                'category': res.meta['category'],
                'title': title,
                'link': link,
                'info': info,
                'hot_index': hot_index
            }

    pass

    def parse(self, res):
        url = "https://top.baidu.com/board?tab="
        urls = [
            {"category": "realtime", "path": f'{url}realtime'},
            {"category": "novel", "path": f'{url}novel'},
            {"category": "movie", "path": f'{url}movie'},
            {"category": "teleplay", "path": f'{url}teleplay'},
            {"category": "cartoon", "path": f'{url}cartoon'},
            {"category": "variety", "path": f'{url}variety'},
            {"category": "documentary", "path": f'{url}documentary'},
            {"category": "star", "path": f'{url}star'},
            {"category": "car", "path": f'{url}car'},
            {"category": "game", "path": f'{url}game'},
        ]
        for url in urls:
            yield scrapy.Request(url=url['path'], meta={'category': url['category']}, headers=util.default_headers,
                                 callback=self.parse_item)
